Q1

First I load the data files

Next I reshape dat into the handful data format. I use tidyr package for the aim instead of reshape package because the foramer is the improved version of the latter.

install.packages('tidyverse')
 URL 'https://cran.rstudio.com/bin/macosx/mavericks/contrib/3.3/tidyverse_1.1.1.tgz' を試しています 
Content type 'application/x-gzip' length 37228 bytes (36 KB)
==================================================
downloaded 36 KB

The downloaded binary packages are in
    /var/folders/1w/nyst5xl16t329h4gpz0j49hm0000gn/T//Rtmp2exVAC/downloaded_packages
library(tidyverse)
Loading tidyverse: ggplot2
Loading tidyverse: tibble
Loading tidyverse: tidyr
Loading tidyverse: readr
Loading tidyverse: purrr
Loading tidyverse: dplyr
Conflicts with tidy packages ----------------------------------------------------------------------------------------------------------------------------------
filter(): dplyr, stats
lag():    dplyr, stats
dat <- dat %>%
tidyr::gather(key=From, value=value, Alabama,Alaska,Arizona,Arkansas,California,Colorado,Connecticut,Delaware,District\ of\ Columbia,Florida,Georgia,Hawaii,Idaho,Illinois,Indiana,Iowa,Kansas,Kentucky,Louisiana,Maine,Maryland,Massachusetts,Michigan,Minnesota,Mississippi,Missouri,Montana,Nebraska,Nevada,New\ Hampshire,New\ Jersey,New\ Mexico,New\ York,North\ Carolina,North\ Dakota,Ohio,Oklahoma,Oregon,Pennsylvania,Rhode\ Island,South\ Carolina,South\ Dakota,Tennessee,Texas,Utah,Vermont,Virginia,Washington,West\ Virginia,Wisconsin,Wyoming)
 エラー:  想定外の入力です  in:
"dat <- dat %>%
tidyr::gather(key=From, value=value, Alabama,Alaska,Arizona,Arkansas,California,Colorado,Connecticut,Delaware,District\"

Delete the puctuations.

Then I add two new columns to the above data frame.

d1 <- merge(dat, states, by.x="From", by.y="States")
d1 <- d1[,c(-5,-6)]
names(d1)[4] <- "ID_From"
d2 <- merge(d1, states, by.x="To", by.y="States")
d2 <- d2[,c(-6,-7)]
names(d2)[5] <- "ID_To"
dat <- d2
dat

Sort the data frame in the ascending order

dat <- arrange(dat, ID_From)
dat <- arrange(dat, ID_To)
dat

Convert the above data frame into the matrix form

Drawing the chord diagram

chorddiag(as.matrix(t5),groupColors=states$Color,showTicks=F,groupnamePadding = 20,groupThickness=.05,groupnameFontsize=10)
row names of the 'data' matrix differ from its column names or the 'groupNames' argument.

Q2

まだindividualを入れてないです。 Read “Stops On Lines” and the all GIS data of bus lines.

library(dplyr)
library(sp)
library(rgdal)
library(leaflet)
library(ggmap)

# Bus Stops
SOL <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/BusStops1216","StopsOnLines1216")
SOL.pj <- spTransform(SOL, CRS("+proj=longlat +datum=WGS84"))

# Bus Routes
CC <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/ComCir1216","ComCir1216")
CC.pj <- spTransform(CC, CRS("+proj=longlat +datum=WGS84"))
LE <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/LimExp1216","LimExp1216")
LE.pj <- spTransform(LE, CRS("+proj=longlat +datum=WGS84"))
LCBD <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/LocalCBD1216","LocalCBD1216")
LCBD.pj <- spTransform(LCBD, CRS("+proj=longlat +datum=WGS84"))
LNCBD <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/LocalNonCBD1216","LocalNonCBD1216")
LNCBD.pj <- spTransform(LNCBD, CRS("+proj=longlat +datum=WGS84"))
RBRT <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/RapidBRT1216","RapidBRT1216")
RBRT.pj <- spTransform(RBRT, CRS("+proj=longlat +datum=WGS84"))

# とりあえずindividual2についてのみreadできることを確認。一旦放置してこれ以外でできるか試す。
I2 <- readOGR("/Users/susu/Desktop/Hong\ Kong/Semester2/Big_Data/assignment_data/as1/Individuals1216","2")
I2.pj <- spTransform(I2, CRS("+proj=longlat +datum=WGS84"))

make Line_list

tmp_CC <- geometry(CC.pj)
tmp_LE <- geometry(LE.pj)
tmp_LCBD <- geometry(LCBD.pj)
tmp_LNCBD <- geometry(LNCBD.pj)
tmp_RBRT <- geometry(RBRT.pj)
tmps <- list(tmp_CC, tmp_LE, tmp_LCBD, tmp_LNCBD, tmp_RBRT)
Line_list <- list()
for (i in 1:5){
  for (j in 1:length(tmps[[i]])){
    Line_list <- c(Line_list, tmps[[i]][j]@lines[[1]]@Lines)
  }
}

make new_id

pjs <- list(CC.pj, LE.pj, LCBD.pj, LNCBD.pj, RBRT.pj)
LinLSs <- list()
for (i in 1:5){
 LinLSs <- c(LinLSs, sapply(pjs[[i]]@lines, function(x) length(x@Lines)))
}
LinLSs <- LinLSs %>% unlist()
new_id <- sapply(1:length(LinLSs), function(x) paste0(x, "_", seq.int(LinLSs[[x]]))) %>% 
  unlist()
SLDF <- mapply(function(x, y) Lines(x, ID = y), x = Line_list, y = new_id) %>%
  list() %>%
  SpatialLines() %>% 
  SpatialLinesDataFrame(data = DAT)
 SpatialLines(.) でエラー: 
  lines list not exclusively filled with Lines objects

make new lines and LA map

Q3

First I load the data.

library(quantmod)
library(highcharter)

x <- getSymbols("AUD/JPY", src = "oanda", auto.assign = FALSE)
y <- getSymbols("GBP/USD", src = "oanda", auto.assign = FALSE)

Next make Bollinger’s bands for each exchange rate.

x.BBands.ll <- BBands(x)$dn
x.BBands.ul <- BBands(x)$up
x.BBands.m <- BBands(x)$mavg
y.BBands.ll <- BBands(y)$dn
y.BBands.ul <- BBands(y)$up
y.BBands.m <- BBands(y)$mavg

The drawing code is as follows.

hc <- highchart(type="stock") %>% 
  hc_title(text="Charting Exchange Rates") %>% 
  hc_subtitle(text = "Data extracted using quantmod package") %>% 
  hc_yAxis_multiples(
    list(top = "0%", height = "50%", offset=0, opposite=TRUE),
    list(top = "50%", height = "50%", offset=0, opposite=TRUE)
  )%>%
  hc_add_series(x, id = "audjpy",name ="audjpy", yAxis=0, color="blue", lineWidth=1.5) %>%
  hc_add_series(x.BBands.ll, id = "audjpy.ll", name="audjpy Lower BBands",yAxis=0,
                color="black",dashStyle='shortdash', lineWidth=1) %>%
  hc_add_series(x.BBands.ul, id = "audjpy.ul", name="audjpy Upper BBands",yAxis=0,
                color="black",lineWidth=1) %>%
  hc_add_series(x.BBands.m, id = "audjpy.m",name="audjpy BBands MA", yAxis=0,
                color="red",lineWidth=1) %>%
  hc_add_series(y, id = "gbpusd",name="gbpusd",yAxis=1, color="green", lineWidth=1.5) %>%
  hc_add_series(y.BBands.ll, id = "gbpusd.ll",name="gbpusd Lower BBands", yAxis=1,
                color="black",dashStyle='shortdash',lineWidth=1) %>%
  hc_add_series(y.BBands.ul, id = "gbpusd.ul",name="gbpusd Upper BBands", yAxis=1,
                color="black",lineWidth=1) %>%
  hc_add_series(y.BBands.m, id = "gbpusd.m",name="gbpusd BBands MA", yAxis=1,
                color="red",lineWidth=1) %>%
  hc_add_theme(hc_theme_538())

hc

Q4

Load libraries and check the raw data. And make ffdf after converting character columns to factor columns in original df.

library(nycflights13)
library(ffbase)
library(ffbase2)
library(biglm)
library(pROC)
library(chron)

tmp <- flights
tmp$carrier <- as.factor(tmp$carrier)
tmp$tailnum <- as.factor(tmp$tailnum)
tmp$origin <- as.factor(tmp$origin)
tmp$dest <- as.factor(tmp$dest)

flightff <- as.ffdf(tmp)

Next I make new columns as follows

flightff$Delay <- ffifelse(flightff$dep_delay > 0, 1,0)
flightff$DepHour <- flightff$hour
flightff$Car <- ffifelse(flightff$carrier %in% as.factor(c("DL","US","DH","UA")), 1, 0)
flightff$Night <- ffifelse(flightff$hour > 18 | flightff$hour < 6, 1, 0)
flightff$Weekend <- ffifelse(day.of.week(month=flightff$month, day=flightff$day, year=flightff$year) == 6, 1, 0)

I exclude the rows whose Delay values are NA and rename it to logitff. And then I split the dataset into train set and test set.

logitff <- flightff[!is.na(flightff$Delay),]

indx <- ff(1:nrow(logitff))
p <- 0.7
trainIndx <- ff(indx[1:trunc(length(indx)*p)])
trainset <- logitff[trainIndx,]
testIndx <- ff(indx[(trunc(length(indx)*p)+1):length(indx)])
testset <- logitff[testIndx,]

Logistic regression

summary(fit)
Large data regression model: bigglm(Delay ~ DepHour + Car + Night + Weekend, data = trainset, 
    family = binomial(), sandwich = TRUE)
Sample size =  229964 
               Coef    (95%     CI)     SE p
(Intercept) -1.8632 -1.8959 -1.8305 0.0164 0
DepHour      0.1108  0.1084  0.1133 0.0012 0
Car         -0.1073 -0.1255 -0.0891 0.0091 0
Night       -0.2672 -0.2968 -0.2376 0.0148 0
Weekend     -0.1737 -0.2025 -0.1450 0.0144 0
Sandwich (model-robust) standard errors

predict and make confusionmatrix in train_set

train_confusion
     
           0      1    Sum
  0   122907  19209 142116
  1    68525  19323  87848
  Sum 191432  38532 229964

predict and make confusionmatrix in test_set

test_confusion
     
          0     1   Sum
  0   50824  7149 57973
  1   31212  9372 40584
  Sum 82036 16521 98557

Draw ROC curve

Q5

First I load the data. And before using spark I delete the irrelevant columns.

Remove the observations satisfying the condition

Split this data into trainset and testset.

test
$test
NA

Use Decision tree

decision_tree <- train %>%
  ml_decision_tree(response="BOROUGH", features = c("LATITUDE","LONGITUDE"), max.bins = 200L, max.depth = 10L, seed=123L) %>%

Prediction

table(pred$BOROUGH, pred$prediction)
               
                    0     1     2     3     4
  BRONX             0     0     0  8876     0
  BROOKLYN      21181     0    24     0     1
  MANHATTAN         0 17944     0     0     0
  QUEENS           25     1 17777     0     0
  STATEN ISLAND    23     0     0     0  3169
LS0tCnRpdGxlOiAiQmlnIERhdGEgQW5hbHl0aWNzIEFzc2lnbm1lbnQgMSIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKIyMgUTEKCkZpcnN0IEkgbG9hZCB0aGUgZGF0YSBmaWxlcwpgYGB7cn0KZGF0IDwtIHJlYWQuY3N2KCIvVXNlcnMvc3VzdS9EZXNrdG9wL0hvbmdcIEtvbmcvU2VtZXN0ZXIyL0JpZ19EYXRhL2Fzc2lnbm1lbnRfZGF0YS9hczEvbWlncmF0aW9uMjAxMi5jc3YiKQpoZWFkKGRhdCkKYGBgCgpgYGB7cn0Kc3RhdGVzIDwtIHJlYWQuY3N2KCIvVXNlcnMvc3VzdS9EZXNrdG9wL0hvbmdcIEtvbmcvU2VtZXN0ZXIyL0JpZ19EYXRhL2Fzc2lnbm1lbnRfZGF0YS9hczEvc3RhdGVzX2Nob3JkLmNzdiIpCnN0YXRlcwpgYGAKCk5leHQgSSByZXNoYXBlIGRhdCBpbnRvIHRoZSBoYW5kZnVsIGRhdGEgZm9ybWF0LiBJIHVzZSAqKnRpZHlyIHBhY2thZ2UqKiBmb3IgdGhlIGFpbSBpbnN0ZWFkIG9mICoqcmVzaGFwZSBwYWNrYWdlKiogYmVjYXVzZSB0aGUgZm9yYW1lciBpcyB0aGUgaW1wcm92ZWQgdmVyc2lvbiBvZiB0aGUgbGF0dGVyLgoKYGBge3J9Cmluc3RhbGwucGFja2FnZXMoJ3RpZHl2ZXJzZScpCmxpYnJhcnkodGlkeXZlcnNlKQpgYGAKCmBgYHtyfQpkYXQgPC0gZGF0ICU+JQp0aWR5cjo6Z2F0aGVyKGtleT1Gcm9tLCB2YWx1ZT12YWx1ZSwgQWxhYmFtYSxBbGFza2EsQXJpem9uYSxBcmthbnNhcyxDYWxpZm9ybmlhLENvbG9yYWRvLENvbm5lY3RpY3V0LERlbGF3YXJlLERpc3RyaWN0Lm9mLkNvbHVtYmlhLEZsb3JpZGEsR2VvcmdpYSxIYXdhaWksSWRhaG8sSWxsaW5vaXMsSW5kaWFuYSxJb3dhLEthbnNhcyxLZW50dWNreSxMb3Vpc2lhbmEsTWFpbmUsTWFyeWxhbmQsTWFzc2FjaHVzZXR0cyxNaWNoaWdhbixNaW5uZXNvdGEsTWlzc2lzc2lwcGksTWlzc291cmksTW9udGFuYSxOZWJyYXNrYSxOZXZhZGEsTmV3LkhhbXBzaGlyZSxOZXcuSmVyc2V5LE5ldy5NZXhpY28sTmV3LllvcmssTm9ydGguQ2Fyb2xpbmEsTm9ydGguRGFrb3RhLE9oaW8sT2tsYWhvbWEsT3JlZ29uLFBlbm5zeWx2YW5pYSxSaG9kZS5Jc2xhbmQsU291dGguQ2Fyb2xpbmEsU291dGguRGFrb3RhLFRlbm5lc3NlZSxUZXhhcyxVdGFoLFZlcm1vbnQsVmlyZ2luaWEsV2FzaGluZ3RvbixXZXN0LlZpcmdpbmlhLFdpc2NvbnNpbixXeW9taW5nKQpgYGAKCkRlbGV0ZSB0aGUgcHVjdHVhdGlvbnMuCgpgYGB7cn0KZGF0IDwtIGRhdGEuZnJhbWUoYXBwbHkoZGF0LCAyLCBmdW5jdGlvbih5KSBnc3ViKCJbWzpwdW5jdDpdXSIsICIgIiwgeSkpKQpkYXQKYGBgCgpUaGVuIEkgYWRkIHR3byBuZXcgY29sdW1ucyB0byB0aGUgYWJvdmUgZGF0YSBmcmFtZS4KCmBgYHtyfQpkMSA8LSBtZXJnZShkYXQsIHN0YXRlcywgYnkueD0iRnJvbSIsIGJ5Lnk9IlN0YXRlcyIpCmQxIDwtIGQxWyxjKC01LC02KV0KbmFtZXMoZDEpWzRdIDwtICJJRF9Gcm9tIgoKZDIgPC0gbWVyZ2UoZDEsIHN0YXRlcywgYnkueD0iVG8iLCBieS55PSJTdGF0ZXMiKQpkMiA8LSBkMlssYygtNiwtNyldCm5hbWVzKGQyKVs1XSA8LSAiSURfVG8iCmRhdCA8LSBkMgpkYXQKYGBgCgpTb3J0IHRoZSBkYXRhIGZyYW1lIGluIHRoZSBhc2NlbmRpbmcgb3JkZXIKCmBgYHtyfQpkYXQgPC0gYXJyYW5nZShkYXQsIElEX0Zyb20pCmRhdCA8LSBhcnJhbmdlKGRhdCwgSURfVG8pCmRhdApgYGAKCkNvbnZlcnQgdGhlIGFib3ZlIGRhdGEgZnJhbWUgaW50byB0aGUgbWF0cml4IGZvcm0KCmBgYHtyfQp0MSA8LSBkYXRbLGMoMSwyLDQsNSldCnQxIDwtIHQxICU+JQogIHNwcmVhZChrZXk9VG8sIHZhbHVlPUlEX1RvKQoKdDIgPC0gZGF0WyxjKDEsMiwzLDQpXQp0MiA8LSB0MiAlPiUKICBzcHJlYWQoa2V5PVRvLCB2YWx1ZT12YWx1ZSkKCnQzIDwtIHJiaW5kKHQxWzEsXSx0MikKdDMkRnJvbSA8LSBhcy5jaGFyYWN0ZXIodDMkRnJvbSkKdDNbMSwyXSA8LSAwCnQzWzEsMV0gPC0gIklEX1RvIgp0MyA8LSB0MyAlPiUKICBhcnJhbmdlKElEX0Zyb20pCgpsaWJyYXJ5KGRhdGEudGFibGUpCnNldGNvbG9yZGVyKHQzLGMoIkZyb20iLCJJRF9Gcm9tIiwiQ29ubmVjdGljdXQiLCJNYWluZSIsIk1hc3NhY2h1c2V0dHMiLCJOZXcgSGFtcHNoaXJlIiwiUmhvZGUgSXNsYW5kIiwKIlZlcm1vbnQiLCJOZXcgSmVyc2V5IiwiTmV3IFlvcmsiLCJQZW5uc3lsdmFuaWEiLCJJbGxpbm9pcyIsIkluZGlhbmEiLAoiTWljaGlnYW4iLCJPaGlvIiwiV2lzY29uc2luIiwiSW93YSIsIkthbnNhcyIsIk1pbm5lc290YSIsCiJNaXNzb3VyaSIsIk5lYnJhc2thIiwiTm9ydGggRGFrb3RhIiwiU291dGggRGFrb3RhIiwiRGVsYXdhcmUiLCJGbG9yaWRhIiwKIkdlb3JnaWEiLCJNYXJ5bGFuZCIsIk5vcnRoIENhcm9saW5hIiwiU291dGggQ2Fyb2xpbmEiLCJWaXJnaW5pYSIsIkRpc3RyaWN0IG9mIENvbHVtYmlhIiwKIldlc3QgVmlyZ2luaWEiLCJBbGFiYW1hIiwiS2VudHVja3kiLCJNaXNzaXNzaXBwaSIsIlRlbm5lc3NlZSIsIkFya2Fuc2FzIiwKIkxvdWlzaWFuYSIsIk9rbGFob21hIiwiVGV4YXMiLCJBcml6b25hIiwiQ29sb3JhZG8iLCJJZGFobyIsCiJNb250YW5hIiwiTmV2YWRhIiwiTmV3IE1leGljbyIsIlV0YWgiLCJXeW9taW5nIiwiQWxhc2thIiwKIkNhbGlmb3JuaWEiLCJIYXdhaWkiLCJPcmVnb24iLCJXYXNoaW5ndG9uIikpCgp0MyA8LSB0M1tjKC0xKSxjKC0yKV0KCnQ0IDwtIHQzWywtMV0Kcm93bmFtZXModDQpIDwtIHQzWywxXQoKdDUgPC0gZGF0YS5mcmFtZShhcHBseSh0NCwgMiwgZnVuY3Rpb24oeSkgYXMubnVtZXJpYyh5KSkpCnJvd25hbWVzKHQ1KSA8LSB0M1ssMV0KdDUKYGBgCgpEcmF3aW5nIHRoZSBjaG9yZCBkaWFncmFtCmBgYHtyfQpsaWJyYXJ5KGNob3JkZGlhZykKCmNob3JkZGlhZyhhcy5tYXRyaXgodDUpLGdyb3VwQ29sb3JzPXN0YXRlcyRDb2xvcixzaG93VGlja3M9Rixncm91cG5hbWVQYWRkaW5nPTIwLGdyb3VwVGhpY2tuZXNzPS4wNSxncm91cG5hbWVGb250c2l6ZT0xMCkKYGBgCgoKIyMgUTIKCuOBvuOBoGluZGl2aWR1YWzjgpLlhaXjgozjgabjgarjgYTjgafjgZnjgIIKUmVhZCAiU3RvcHMgT24gTGluZXMiIGFuZCB0aGUgYWxsIEdJUyBkYXRhIG9mIGJ1cyBsaW5lcy4KYGBge3J9CmxpYnJhcnkoZHBseXIpCmxpYnJhcnkoc3ApCmxpYnJhcnkocmdkYWwpCmxpYnJhcnkobGVhZmxldCkKbGlicmFyeShnZ21hcCkKCiMgQnVzIFN0b3BzClNPTCA8LSByZWFkT0dSKCIvVXNlcnMvc3VzdS9EZXNrdG9wL0hvbmdcIEtvbmcvU2VtZXN0ZXIyL0JpZ19EYXRhL2Fzc2lnbm1lbnRfZGF0YS9hczEvQnVzU3RvcHMxMjE2IiwiU3RvcHNPbkxpbmVzMTIxNiIpClNPTC5waiA8LSBzcFRyYW5zZm9ybShTT0wsIENSUygiK3Byb2o9bG9uZ2xhdCArZGF0dW09V0dTODQiKSkKCiMgQnVzIFJvdXRlcwpDQyA8LSByZWFkT0dSKCIvVXNlcnMvc3VzdS9EZXNrdG9wL0hvbmdcIEtvbmcvU2VtZXN0ZXIyL0JpZ19EYXRhL2Fzc2lnbm1lbnRfZGF0YS9hczEvQ29tQ2lyMTIxNiIsIkNvbUNpcjEyMTYiKQpDQy5waiA8LSBzcFRyYW5zZm9ybShDQywgQ1JTKCIrcHJvaj1sb25nbGF0ICtkYXR1bT1XR1M4NCIpKQpMRSA8LSByZWFkT0dSKCIvVXNlcnMvc3VzdS9EZXNrdG9wL0hvbmdcIEtvbmcvU2VtZXN0ZXIyL0JpZ19EYXRhL2Fzc2lnbm1lbnRfZGF0YS9hczEvTGltRXhwMTIxNiIsIkxpbUV4cDEyMTYiKQpMRS5waiA8LSBzcFRyYW5zZm9ybShMRSwgQ1JTKCIrcHJvaj1sb25nbGF0ICtkYXR1bT1XR1M4NCIpKQpMQ0JEIDwtIHJlYWRPR1IoIi9Vc2Vycy9zdXN1L0Rlc2t0b3AvSG9uZ1wgS29uZy9TZW1lc3RlcjIvQmlnX0RhdGEvYXNzaWdubWVudF9kYXRhL2FzMS9Mb2NhbENCRDEyMTYiLCJMb2NhbENCRDEyMTYiKQpMQ0JELnBqIDwtIHNwVHJhbnNmb3JtKExDQkQsIENSUygiK3Byb2o9bG9uZ2xhdCArZGF0dW09V0dTODQiKSkKTE5DQkQgPC0gcmVhZE9HUigiL1VzZXJzL3N1c3UvRGVza3RvcC9Ib25nXCBLb25nL1NlbWVzdGVyMi9CaWdfRGF0YS9hc3NpZ25tZW50X2RhdGEvYXMxL0xvY2FsTm9uQ0JEMTIxNiIsIkxvY2FsTm9uQ0JEMTIxNiIpCkxOQ0JELnBqIDwtIHNwVHJhbnNmb3JtKExOQ0JELCBDUlMoIitwcm9qPWxvbmdsYXQgK2RhdHVtPVdHUzg0IikpClJCUlQgPC0gcmVhZE9HUigiL1VzZXJzL3N1c3UvRGVza3RvcC9Ib25nXCBLb25nL1NlbWVzdGVyMi9CaWdfRGF0YS9hc3NpZ25tZW50X2RhdGEvYXMxL1JhcGlkQlJUMTIxNiIsIlJhcGlkQlJUMTIxNiIpClJCUlQucGogPC0gc3BUcmFuc2Zvcm0oUkJSVCwgQ1JTKCIrcHJvaj1sb25nbGF0ICtkYXR1bT1XR1M4NCIpKQoKIyDjgajjgorjgYLjgYjjgZppbmRpdmlkdWFsMuOBq+OBpOOBhOOBpuOBruOBv3JlYWTjgafjgY3jgovjgZPjgajjgpLnorroqo3jgILkuIDml6bmlL7nva7jgZfjgabjgZPjgozku6XlpJbjgafjgafjgY3jgovjgYvoqabjgZnjgIIKSTIgPC0gcmVhZE9HUigiL1VzZXJzL3N1c3UvRGVza3RvcC9Ib25nXCBLb25nL1NlbWVzdGVyMi9CaWdfRGF0YS9hc3NpZ25tZW50X2RhdGEvYXMxL0luZGl2aWR1YWxzMTIxNiIsIjIiKQpJMi5waiA8LSBzcFRyYW5zZm9ybShJMiwgQ1JTKCIrcHJvaj1sb25nbGF0ICtkYXR1bT1XR1M4NCIpKQpgYGAKCm1ha2UgTGluZV9saXN0CmBgYHtyfQp0bXBfQ0MgPC0gZ2VvbWV0cnkoQ0MucGopCnRtcF9MRSA8LSBnZW9tZXRyeShMRS5waikKdG1wX0xDQkQgPC0gZ2VvbWV0cnkoTENCRC5waikKdG1wX0xOQ0JEIDwtIGdlb21ldHJ5KExOQ0JELnBqKQp0bXBfUkJSVCA8LSBnZW9tZXRyeShSQlJULnBqKQp0bXBzIDwtIGxpc3QodG1wX0NDLCB0bXBfTEUsIHRtcF9MQ0JELCB0bXBfTE5DQkQsIHRtcF9SQlJUKQoKTGluZV9saXN0IDwtIGxpc3QoKQpmb3IgKGkgaW4gMTo1KXsKICBmb3IgKGogaW4gMTpsZW5ndGgodG1wc1tbaV1dKSl7CiAgICBMaW5lX2xpc3QgPC0gYyhMaW5lX2xpc3QsIHRtcHNbW2ldXVtqXUBsaW5lc1tbMV1dQExpbmVzKQogIH0KfQpgYGAKCgptYWtlIG5ld19pZApgYGB7cn0KcGpzIDwtIGxpc3QoQ0MucGosIExFLnBqLCBMQ0JELnBqLCBMTkNCRC5waiwgUkJSVC5waikKTGluTFNzIDwtIGxpc3QoKQpmb3IgKGkgaW4gMTo1KXsKIExpbkxTcyA8LSBjKExpbkxTcywgc2FwcGx5KHBqc1tbaV1dQGxpbmVzLCBmdW5jdGlvbih4KSBsZW5ndGgoeEBMaW5lcykpKQp9CkxpbkxTcyA8LSBMaW5MU3MgJT4lIHVubGlzdCgpCgpuZXdfaWQgPC0gc2FwcGx5KDE6bGVuZ3RoKExpbkxTcyksIGZ1bmN0aW9uKHgpIHBhc3RlMCh4LCAiXyIsIHNlcS5pbnQoTGluTFNzW1t4XV0pKSkgJT4lIAogIHVubGlzdCgpCmBgYAoKYGBge3J9CiMjIG1ha2UgYSBuZXcgZGF0YS5mcmFtZSAob25seSByb3V0ZV9pZCkKREFUPWRhdGEuZnJhbWUobWF0cml4KHJlcChOQSwxKSxucm93PTEpKVstMSxdCmZvciAoaSBpbiAxOjUpewogIGRmIDwtIGRhdGEuZnJhbWUocm91dGVfaWQgPSBwanNbW2ldXUBkYXRhJFZBUl9JREVOVCkKICBEQVQgPC0gcmJpbmQoREFULCBkZikKfQpyb3duYW1lcyhEQVQpIDwtIG5ld19pZAoKU0xERiA8LSBtYXBwbHkoZnVuY3Rpb24oeCwgeSkgTGluZXMoeCwgSUQgPSB5KSwgeCA9IExpbmVfbGlzdCwgeSA9IG5ld19pZCkgJT4lCiAgI2xpc3QoKSAlPiUKICBTcGF0aWFsTGluZXMoKSAlPiUgCiAgU3BhdGlhbExpbmVzRGF0YUZyYW1lKGRhdGEgPSBEQVQpCmBgYAoKCm1ha2UgbmV3IGxpbmVzIGFuZCBMQSBtYXAKYGBge3J9CmRhdCA8LSBnZW9jb2RlKCdMb3MgQW5nZWxzJykKCmxlYWZsZXQoKSAlPiUKICBzZXRWaWV3KGxuZyA9IGRhdFsnbG9uJ10sIGxhdCA9IGRhdFsnbGF0J10JLCB6b29tID0gMTEpICU+JQogIGFkZFBvbHlsaW5lcyhkYXRhID0gU0xERiwgY29sb3IgPSAiYmxhY2siLCBvcGFjaXR5ID0gMSwgd2VpZ2h0ID0gMSkgJT4lIAogIGFkZENpcmNsZXMoZGF0YT1TT0wucGpAZGF0YSx+TE9ORywgfkxBVCwgY29sb3IgPSAicmVkIiwgd2VpZ2h0ID0gMC4zKSAlPiUKICBhZGRUaWxlcygpCmBgYAoKCiMjIFEzCgpGaXJzdCBJIGxvYWQgdGhlIGRhdGEuCmBgYHtyfQpsaWJyYXJ5KHF1YW50bW9kKQpsaWJyYXJ5KGhpZ2hjaGFydGVyKQoKeCA8LSBnZXRTeW1ib2xzKCJBVUQvSlBZIiwgc3JjID0gIm9hbmRhIiwgYXV0by5hc3NpZ24gPSBGQUxTRSkKeSA8LSBnZXRTeW1ib2xzKCJHQlAvVVNEIiwgc3JjID0gIm9hbmRhIiwgYXV0by5hc3NpZ24gPSBGQUxTRSkKYGBgCgpOZXh0IG1ha2UgQm9sbGluZ2VyJ3MgYmFuZHMgZm9yIGVhY2ggZXhjaGFuZ2UgcmF0ZS4KYGBge3J9CnguQkJhbmRzLmxsIDwtIEJCYW5kcyh4KSRkbgp4LkJCYW5kcy51bCA8LSBCQmFuZHMoeCkkdXAKeC5CQmFuZHMubSA8LSBCQmFuZHMoeCkkbWF2Zwp5LkJCYW5kcy5sbCA8LSBCQmFuZHMoeSkkZG4KeS5CQmFuZHMudWwgPC0gQkJhbmRzKHkpJHVwCnkuQkJhbmRzLm0gPC0gQkJhbmRzKHkpJG1hdmcKYGBgCgpUaGUgZHJhd2luZyBjb2RlIGlzIGFzIGZvbGxvd3MuCmBgYHtyfQpoYyA8LSBoaWdoY2hhcnQodHlwZT0ic3RvY2siKSAlPiUgCiAgaGNfdGl0bGUodGV4dD0iQ2hhcnRpbmcgRXhjaGFuZ2UgUmF0ZXMiKSAlPiUgCiAgaGNfc3VidGl0bGUodGV4dCA9ICJEYXRhIGV4dHJhY3RlZCB1c2luZyBxdWFudG1vZCBwYWNrYWdlIikgJT4lIAogIGhjX3lBeGlzX211bHRpcGxlcygKICAgIGxpc3QodG9wID0gIjAlIiwgaGVpZ2h0ID0gIjUwJSIsIG9mZnNldD0wLCBvcHBvc2l0ZT1UUlVFKSwKICAgIGxpc3QodG9wID0gIjUwJSIsIGhlaWdodCA9ICI1MCUiLCBvZmZzZXQ9MCwgb3Bwb3NpdGU9VFJVRSkKICApJT4lCiAgaGNfYWRkX3Nlcmllcyh4LCBpZCA9ICJhdWRqcHkiLG5hbWUgPSJhdWRqcHkiLCB5QXhpcz0wLCBjb2xvcj0iYmx1ZSIsIGxpbmVXaWR0aD0xLjUpICU+JQogIGhjX2FkZF9zZXJpZXMoeC5CQmFuZHMubGwsIGlkID0gImF1ZGpweS5sbCIsIG5hbWU9ImF1ZGpweSBMb3dlciBCQmFuZHMiLHlBeGlzPTAsCiAgICAgICAgICAgICAgICBjb2xvcj0iYmxhY2siLGRhc2hTdHlsZT0nc2hvcnRkYXNoJywgbGluZVdpZHRoPTEpICU+JQogIGhjX2FkZF9zZXJpZXMoeC5CQmFuZHMudWwsIGlkID0gImF1ZGpweS51bCIsIG5hbWU9ImF1ZGpweSBVcHBlciBCQmFuZHMiLHlBeGlzPTAsCiAgICAgICAgICAgICAgICBjb2xvcj0iYmxhY2siLGxpbmVXaWR0aD0xKSAlPiUKICBoY19hZGRfc2VyaWVzKHguQkJhbmRzLm0sIGlkID0gImF1ZGpweS5tIixuYW1lPSJhdWRqcHkgQkJhbmRzIE1BIiwgeUF4aXM9MCwKICAgICAgICAgICAgICAgIGNvbG9yPSJyZWQiLGxpbmVXaWR0aD0xKSAlPiUKICBoY19hZGRfc2VyaWVzKHksIGlkID0gImdicHVzZCIsbmFtZT0iZ2JwdXNkIix5QXhpcz0xLCBjb2xvcj0iZ3JlZW4iLCBsaW5lV2lkdGg9MS41KSAlPiUKICBoY19hZGRfc2VyaWVzKHkuQkJhbmRzLmxsLCBpZCA9ICJnYnB1c2QubGwiLG5hbWU9ImdicHVzZCBMb3dlciBCQmFuZHMiLCB5QXhpcz0xLAogICAgICAgICAgICAgICAgY29sb3I9ImJsYWNrIixkYXNoU3R5bGU9J3Nob3J0ZGFzaCcsbGluZVdpZHRoPTEpICU+JQogIGhjX2FkZF9zZXJpZXMoeS5CQmFuZHMudWwsIGlkID0gImdicHVzZC51bCIsbmFtZT0iZ2JwdXNkIFVwcGVyIEJCYW5kcyIsIHlBeGlzPTEsCiAgICAgICAgICAgICAgICBjb2xvcj0iYmxhY2siLGxpbmVXaWR0aD0xKSAlPiUKICBoY19hZGRfc2VyaWVzKHkuQkJhbmRzLm0sIGlkID0gImdicHVzZC5tIixuYW1lPSJnYnB1c2QgQkJhbmRzIE1BIiwgeUF4aXM9MSwKICAgICAgICAgICAgICAgIGNvbG9yPSJyZWQiLGxpbmVXaWR0aD0xKSAlPiUKICBoY19hZGRfdGhlbWUoaGNfdGhlbWVfNTM4KCkpCgpoYwpgYGAKCgojIyBRNAoKTG9hZCBsaWJyYXJpZXMgYW5kIGNoZWNrIHRoZSByYXcgZGF0YS4gQW5kIG1ha2UgZmZkZiBhZnRlciBjb252ZXJ0aW5nIGNoYXJhY3RlciBjb2x1bW5zIHRvIGZhY3RvciBjb2x1bW5zIGluIG9yaWdpbmFsIGRmLgpgYGB7cn0KbGlicmFyeShueWNmbGlnaHRzMTMpCmxpYnJhcnkoZmZiYXNlKQpsaWJyYXJ5KGZmYmFzZTIpCmxpYnJhcnkoYmlnbG0pCmxpYnJhcnkocFJPQykKbGlicmFyeShjaHJvbikKCnRtcCA8LSBmbGlnaHRzCnRtcCRjYXJyaWVyIDwtIGFzLmZhY3Rvcih0bXAkY2FycmllcikKdG1wJHRhaWxudW0gPC0gYXMuZmFjdG9yKHRtcCR0YWlsbnVtKQp0bXAkb3JpZ2luIDwtIGFzLmZhY3Rvcih0bXAkb3JpZ2luKQp0bXAkZGVzdCA8LSBhcy5mYWN0b3IodG1wJGRlc3QpCgpmbGlnaHRmZiA8LSBhcy5mZmRmKHRtcCkKYGBgCgpOZXh0IEkgbWFrZSBuZXcgY29sdW1ucyBhcyBmb2xsb3dzCmBgYHtyfQpmbGlnaHRmZiREZWxheSA8LSBmZmlmZWxzZShmbGlnaHRmZiRkZXBfZGVsYXkgPiAwLCAxLDApCmZsaWdodGZmJERlcEhvdXIgPC0gZmxpZ2h0ZmYkaG91cgpmbGlnaHRmZiRDYXIgPC0gZmZpZmVsc2UoZmxpZ2h0ZmYkY2FycmllciAlaW4lIGFzLmZhY3RvcihjKCJETCIsIlVTIiwiREgiLCJVQSIpKSwgMSwgMCkKZmxpZ2h0ZmYkTmlnaHQgPC0gZmZpZmVsc2UoZmxpZ2h0ZmYkaG91ciA+IDE4IHwgZmxpZ2h0ZmYkaG91ciA8IDYsIDEsIDApCmZsaWdodGZmJFdlZWtlbmQgPC0gZmZpZmVsc2UoZGF5Lm9mLndlZWsobW9udGg9ZmxpZ2h0ZmYkbW9udGgsIGRheT1mbGlnaHRmZiRkYXksIHllYXI9ZmxpZ2h0ZmYkeWVhcikgPT0gNiwgMSwgMCkKYGBgCgpJIGV4Y2x1ZGUgdGhlIHJvd3Mgd2hvc2UgRGVsYXkgdmFsdWVzIGFyZSBOQSBhbmQgcmVuYW1lIGl0IHRvIGxvZ2l0ZmYuCkFuZCB0aGVuIEkgc3BsaXQgdGhlIGRhdGFzZXQgaW50byB0cmFpbiBzZXQgYW5kIHRlc3Qgc2V0LgpgYGB7cn0KbG9naXRmZiA8LSBmbGlnaHRmZlshaXMubmEoZmxpZ2h0ZmYkRGVsYXkpLF0KCmluZHggPC0gZmYoMTpucm93KGxvZ2l0ZmYpKQpwIDwtIDAuNwp0cmFpbkluZHggPC0gZmYoaW5keFsxOnRydW5jKGxlbmd0aChpbmR4KSpwKV0pCnRyYWluc2V0IDwtIGxvZ2l0ZmZbdHJhaW5JbmR4LF0KdGVzdEluZHggPC0gZmYoaW5keFsodHJ1bmMobGVuZ3RoKGluZHgpKnApKzEpOmxlbmd0aChpbmR4KV0pCnRlc3RzZXQgPC0gbG9naXRmZlt0ZXN0SW5keCxdCmBgYAoKTG9naXN0aWMgcmVncmVzc2lvbiAKYGBge3J9CmZpdCA8LSBiaWdnbG0uZmZkZihEZWxheX5EZXBIb3VyK0NhcitOaWdodCtXZWVrZW5kLCBkYXRhID0gdHJhaW5zZXQsIGZhbWlseT1iaW5vbWlhbCgpLCBzYW5kd2ljaD1UUlVFKQpzdW1tYXJ5KGZpdCkKYGBgCgpwcmVkaWN0IGFuZCBtYWtlIGNvbmZ1c2lvbm1hdHJpeCBpbiB0cmFpbl9zZXQKYGBge3J9CnRyYWluX3ByZWQgPC0gcHJlZGljdChmaXQsIG5ld2RhdGEgPSB0cmFpbnNldCwgdHlwZT0icmVzcG9uc2UiKQp0cmFpbl9wcmVkIDwtIGlmZWxzZSh0cmFpbl9wcmVkPjAuNSwgMSwwKQp0cmFpbl9jb25mdXNpb24gPC0gdGFibGUoYXMuaW50ZWdlcihhcy5kYXRhLmZyYW1lKHRyYWluc2V0KSREZWxheSksIGFzLmludGVnZXIodHJhaW5fcHJlZCkpCnRyYWluX2NvbmZ1c2lvbiA8LSBhZGRtYXJnaW5zKHRyYWluX2NvbmZ1c2lvbikKdHJhaW5fY29uZnVzaW9uCmBgYAoKcHJlZGljdCBhbmQgbWFrZSBjb25mdXNpb25tYXRyaXggaW4gdGVzdF9zZXQKYGBge3J9CnRlc3RfcHJlZCA8LSBwcmVkaWN0KGZpdCwgbmV3ZGF0YSA9IHRlc3RzZXQsIHR5cGU9InJlc3BvbnNlIikKdGVzdF9wcmVkIDwtIGlmZWxzZSh0ZXN0X3ByZWQ+MC41LCAxLDApCnRlc3RfY29uZnVzaW9uIDwtIHRhYmxlKGFzLmludGVnZXIoYXMuZGF0YS5mcmFtZSh0ZXN0c2V0KSREZWxheSksIGFzLmludGVnZXIodGVzdF9wcmVkKSkKdGVzdF9jb25mdXNpb24gPC0gYWRkbWFyZ2lucyh0ZXN0X2NvbmZ1c2lvbikKdGVzdF9jb25mdXNpb24KYGBgCgpEcmF3IFJPQyBjdXJ2ZQpgYGB7cn0KdGVzdF9wcmVkIDwtIHByZWRpY3QoZml0LCBuZXdkYXRhID0gdGVzdHNldCwgdHlwZT0icmVzcG9uc2UiKQpyb2MgPC0gcm9jKGFzLmludGVnZXIoYXMuZGF0YS5mcmFtZSh0ZXN0c2V0KSREZWxheSksIGFzLm51bWVyaWModGVzdF9wcmVkKSkKcGxvdChyb2MpCmBgYAoKCiMjIFE1CgpGaXJzdCBJIGxvYWQgdGhlIGRhdGEuIEFuZCBiZWZvcmUgdXNpbmcgc3BhcmsgSSBkZWxldGUgdGhlIGlycmVsZXZhbnQgY29sdW1ucy4KYGBge3J9CmxpYnJhcnkoc3BhcmtseXIpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkocmVhZHIpCnNjIDwtIHNwYXJrX2Nvbm5lY3QobWFzdGVyID0gImxvY2FsIikKCiMg5YWD44GuY3N244Gr5ZWP6aGM44GM44GC44KL44Gj44G944GE77yfIOKGkiBVbnNwZWNpZmllZOOBjOaAkuOCieOCjOOBpuOCi+OBo+OBveOBhOOAgOKGkuOAgFVuc3BlY2lmaWVk44KSMOOBq+WkieOBiOOCi+OBi+OAguKGkuS7iuW6puOBr1BBU1NFTkdFUiBWRUhJQ0tF44GM5oCS44KJ44KM44KL44CCCiMg6KaB44GZ44KL44GrU3RyaW5n44GM5YWo6YOo44OA44Oh44Gj44G944GE44CC44Gn44KCdGl0YW5pY+OBp+OBr1N0cmluZ+OCguS4iuaJi+OBj+a4oeOBm+OBpuOCi+OAguOBquOBnO+8nwojIOS6jOOBpOOBruS+i+OBi+OCieepuueZveOBr+WIpeOBq+WVj+mhjOOBp+OBr+OBquOBhOOBk+OBqOOBjOOCj+OBi+OCi+OAggpkYXQgPC0gcmVhZF9jc3YoIi9Vc2Vycy9zdXN1L0Rlc2t0b3AvSG9uZ1wgS29uZy9TZW1lc3RlcjIvQmlnX0RhdGEvYXNzaWdubWVudF9kYXRhL2FzMS9OWVBEX01vdG9yX1ZlaGljbGVfQ29sbGlzaW9ucy5jc3YiKQpkYXQgPC0gZGF0WyxjKCJCT1JPVUdIIiwiTEFUSVRVREUiLCJMT05HSVRVREUiLCJVTklRVUUgS0VZIildCm55cGRfdGJsIDwtIGNvcHlfdG8oc2MsIGRhdCwgIm55cGRfdGJsIixvdmVyd3JpdGUgPSBUUlVFKQpgYGAKClJlbW92ZSB0aGUgb2JzZXJ2YXRpb25zIHNhdGlzZnlpbmcgdGhlIGNvbmRpdGlvbgpgYGB7cn0KbnlwZF90YmwgPC0gbnlwZF90YmwgJT4lCiAgZmlsdGVyKEJPUk9VR0ghPSIiLCFpcy5uYShMQVRJVFVERSksIWlzLm5hKExPTkdJVFVERSksTEFUSVRVREUhPTAsTE9OR0lUVURFIT0wKQpueXBkX3RibApgYGAKClNwbGl0IHRoaXMgZGF0YSBpbnRvIHRyYWluc2V0IGFuZCB0ZXN0c2V0LgpgYGB7cn0KcGFydGl0aW9ucyA8LSBueXBkX3RibCAlPiUKICBzZGZfcGFydGl0aW9uKHRyYWluaW5nID0gMC45LCB0ZXN0ID0gMC4xLCBzZWVkID0gMTIzKQp0cmFpbiA8LSBwYXJ0aXRpb25zWzFdJHRyYWluaW5nCnRlc3QgPC0gcGFydGl0aW9uc1syXSR0ZXN0CmBgYAoKVXNlIERlY2lzaW9uIHRyZWUKYGBge3J9CmRlY2lzaW9uX3RyZWUgPC0gdHJhaW4gJT4lCiAgbWxfZGVjaXNpb25fdHJlZShyZXNwb25zZT0iQk9ST1VHSCIsIGZlYXR1cmVzID0gYygiTEFUSVRVREUiLCJMT05HSVRVREUiKSwgbWF4LmJpbnMgPSAyMDBMLCBtYXguZGVwdGggPSAxMEwsIHNlZWQ9MTIzTCkgJT4lCmBgYAoKUHJlZGljdGlvbgpgYGB7cn0KcHJlZCA8LSBzZGZfcHJlZGljdChkZWNpc2lvbl90cmVlLCB0ZXN0KSAlPiUKICBjb2xsZWN0Cgp0YWJsZShwcmVkJEJPUk9VR0gsIHByZWQkcHJlZGljdGlvbikKYGBgCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCg==